/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package LinkRetriever;

import General.Globals;
import General.RequestMaker;
import General.Utilities;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;

/**
 *
 * @author Manuel
 */
public class SiteParser {
    
    private int siteType;
    private int elementCounter;
    private ListManager mgr;


    public SiteParser(ListManager m){
        elementCounter=0;
        mgr = m;
    }


    //extracts all links from a page and adds them to the proper collections, filtering them according to the sitetype
    public void parse(String url) throws MalformedURLException{
        siteType = checkSiteType(url);
        if(siteType == -1)
            return;
        String page = RequestMaker.getPage(url);
        ArrayList<String> links = extractRelevantLinks(page,url);
        String ref;
        for (int i = 0;i<links.size();i++){
            ref = links.get(i);

            //filter projects according to google's properties
            if(siteType == Globals.SITE_TYPE_GOOGLE){
                    if(filterGoogleCodeProjects(ref)){
                        if(mgr.addProjectLink(ref))
                            elementCounter++;
                    }
                    else
                        mgr.insertIfNotVisited(ref);
                }

            //filter according to sourceforge properties
            if(siteType == Globals.SITE_TYPE_SOURCEFORGE){
                if(filterSourceForgeProjects(ref)){
                    if(mgr.addProjectLink(ref))
                        elementCounter++;
                }
              else{
                     mgr.insertIfNotVisited(ref);
               }
                
            }


            if(elementCounter>=Globals.crawlLimit){
                mgr.deleteAll();
                break;
            }
        }
    }

    //extracts all normalized links (hrefs) given a page, a base url and the type of the site
    public ArrayList<String> extractRelevantLinks(String page, String baseURL) throws MalformedURLException{
        URL base;
        base = new URL(baseURL);
        ArrayList<String> links = new ArrayList();
        int nextLink = page.indexOf("href=");
        String currentLink = "";
        page = page.toLowerCase();

        while(nextLink >= 0){
            int nextRef = page.indexOf("\"", nextLink) + 1;
            int endRef = page.indexOf("\"", nextRef);
            if(nextRef < 0 || endRef < 0)
                break;
            currentLink = normalize(page.substring(nextRef,endRef),base);
            if(currentLink.equals(""))
                break;

            //Filter relevant links which can contain projects or which are projects
            if((siteType == Globals.SITE_TYPE_GOOGLE && filterGoogleCodeSites(currentLink))||filterGoogleCodeProjects(currentLink))
                links.add(currentLink);

            if(siteType == Globals.SITE_TYPE_SOURCEFORGE && this.filterSourceForgeSites(currentLink) || this.filterSourceForgeProjects(currentLink)){
                links.add(currentLink);
            }
            page = page.substring(endRef,page.length()-1);
            nextLink = page.indexOf("href=");
        }
        return links;
    }


    //normalizes a link by making it proper i.e. no escape sequences or relative links, all to lowercase
    private String normalize(String toNormalize, URL base) {

            if(!(toNormalize.contains("http://") || (toNormalize.contains("https://")))){
            try {
                toNormalize = (new URL(base, toNormalize)).toString();
            } catch (Exception ex) {
                System.out.println("Could not normalize URL");
                return "";
            }
            }

            if(toNormalize.contains("#"))
                toNormalize = toNormalize.substring(0,toNormalize.indexOf("#"));


            toNormalize = toNormalize.replaceAll("&amp;", "&");

            return toNormalize;
    }


    //returns true if the link correspondes to a project in one of the relevant languages (defined in Globals)
    private boolean checkIfCorrectLanguage(String link){

        boolean jsAllowed = false;
        boolean cppAllowed = false;
        
        //in order to avoid taking ambiguous languages which are not wanted (e.g. javascript because it contains "java")
        for(int i = 0;i<Globals.languages.length;i++){
            if(Globals.languages[i].equals("javascript"))
                jsAllowed = true;
            if(Globals.languages[i].equals("cplusplus"))
                cppAllowed = true;
        }
        
        if(!jsAllowed && link.contains("javascript"))
              return false;

        if(!cppAllowed && link.contains("cplusplus"))
              return false;



        for(int i = 0;i<Globals.languages.length;i++){
            if(link.contains(Globals.languages[i])){
                return true;
            }
        }

        return false;
    }

    
    private int checkSiteType(String url) {
        if(url.toLowerCase().contains("google")){
            return Globals.SITE_TYPE_GOOGLE;
        }
        if(url.toLowerCase().contains("sourceforge")){
            return Globals.SITE_TYPE_SOURCEFORGE;
        }

        else
            return -1;
    }


 //------------------------------------- Google Code Filter ---------------------------------

    //returns true if a link potentially contains google code projects
    private boolean filterGoogleCodeSites(String link){
        //TODO normalize link
        String upLink = link.toLowerCase();
        boolean rightLanguage = checkIfCorrectLanguage(upLink);
        if(upLink.contains("http://code.google.com/hosting/search?") && rightLanguage)
            return true;
        return false;
    }


    //returns true if a link is a google code project
    private boolean filterGoogleCodeProjects(String link){
        if(link.startsWith("http://code.google.com/p/") && !link.contains("http://code.google.com/p/support/"))
            return true;

        else
            return false;
    }

 //-------------------------------------- Other Website Filter ------------------------------

    private boolean filterSourceForgeSites(String link){
        //TODO normalize link
        String upLink = link.toLowerCase();
        if(upLink.contains("http://sourceforge.net/directory/")){
            return true;
        }
        return false;
    }
    private boolean filterSourceForgeProjects(String link) {
        //TODO normalize link

        if(link.startsWith("http://sourceforge.net/projects/") && link.endsWith("/") && (link.replaceAll("[^/]","").length() == 5 ))
            return true;

        else
            return false;
    }








}
